# -*- coding: utf-8 -*-
'''
Created on Oct 8, 2013

@author: LONG HOANG GIANG
'''
from urlparse import urljoin
import logging
import os
import sys
import traceback
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import lib
import json
from database.mysql import MySQL
import datetime
import time
import workerpool

logging.basicConfig(level=logging.DEBUG, format='%(levelname)s :: %(asctime)s :: %(message)s', datefmt='%d/%m/%Y %H:%M:%S')


class WorkerJob(workerpool.Job):
    
    def __init__(self, func, chapter, path):
        self.func = func
        self.chapter = chapter
        self.path = path

    def run(self):
        self.func(self.chapter, self.path)
    

class TruyenNgonTinh():
    
    catFolder = ""
    
    def getDetail(self, chapter, path):
        try:
            if not path.endswith('/'): path += '/'
            print '> get detail of {0} with url {1}'.format(chapter['name'], chapter['url'])
            tree = lib.Web.load(chapter['url']).build_tree()
            xpathList = ["//div[@class='entry']", "//div[@class='entry-content']//div[@style='display: none']/preceding-sibling::*[1]"]
            for iXpath in xpathList:
                contentNode = tree.xpath(iXpath)
                if len(contentNode) == 0: continue
                lib.Etree.cleanNode("./div[@class='ann']", contentNode)
                lib.Etree.cleanNodeNextSibling(contentNode[0].xpath(".//div[@style='display: none']"), True)
                content = lib.html2text(contentNode[0], chapter['url'])
                print content
                content = '''<h4>{0}</h4><div id="content">{1}</div>'''.format(chapter['name'], content)
                
                lib.file_put_content(json.dumps(content), path + "{0}.txt".format(chapter['path']))
                
                return content
        except:
            logging.error(chapter['name'] + "- " + chapter['url'])
            traceback.print_exc()
        return None
    
    def getChapters(self, url):
        logging.info("getChapters - url: {0}".format(url))
        data = [{'name': 'GIỚI THIỆU', 'url': url, 'path': 'gioi-thieu'}]
        tree = lib.Web.load(url).build_tree()
        nodes = tree.xpath("//div/h3[contains(., 'Đọc từ đầu')]/following-sibling::*[1]/li/a".decode('utf-8'))
        if len(nodes) == 0:
            lib.Web.removeCache(url)
            tree = lib.Web.load(url).build_tree()
        for node in nodes:
            name = lib.stringify(node)
            link = node.get('href')
            if name == '' or link == '': continue
            name = lib.toUpper(name)
            link = urljoin(url, link)
            data.append({'name': name, 'url': link, 'path': lib.crc32unsigned(link)})
            
        for i in data:
            print i['name'], i['url']
        
        return data
    
    def getDescription(self, contentNode):
        xpathList = [".//div[@class='entry']", ".//div[@class='entry-content']//div[@style='display: none']/preceding-sibling::*[1]"]
        for iXpath in xpathList:
            node = contentNode.xpath(iXpath)
            if len(node) == 0: continue
            lib.Etree.cleanNode("./div[@class='ann']", contentNode)
            lib.Etree.cleanNodeNextSibling(contentNode[0].xpath(".//div[@style='display: none']"), True)
            content = lib.html2text(node[0], textmode=True, ignore_images=True)
            return content
    
    def getAStory(self, url):
        tree = lib.Web.load(url).build_tree()
        data = {'thumbnail': '', 'tag': '', 'state': '', 'path': lib.md5(url), 'chapters': [], 'desc': ''}
        contentNode = tree.xpath("//div[@id='content']")
        if len(contentNode) == 0: return None
        thumbnailNode = contentNode[0].xpath(".//img[contains(@class, 'post_thumbnail')]")
        if len(thumbnailNode) > 0:
            thumb = thumbnailNode[0].get('src')
            if thumb != '':
                thumb = urljoin(url, thumb)
                data['thumbnail'] = thumb
        tagNodes = contentNode[0].xpath(".//div[@class='postdate' and contains(., 'Thể Loại')]/a[@rel='tag']".decode('utf-8'))
        tag = []
        for node in tagNodes:
            itag = lib.stringify(node)
            if itag != '':
                tag.append(itag)
        data['tag'] = ', '.join(tag)
        # TÌNH TRẠNG TRUYỆN
        
        ttNode = contentNode[0].xpath(".//div[@class='postdate' and contains(., 'Tình Trạng')]/a".decode('utf-8'))
        if len(ttNode) > 0:
            tinhtrang = lib.stringify(ttNode)
            if tinhtrang == '': tinhtrang = 'Hoàn thành'
            data['state'] = tinhtrang
            
        description = self.getDescription(contentNode[0])
        data['desc'] = description
        
        # GET LIST CHAPTER
        nodes = tree.xpath("//div/h3[contains(., 'Đọc từ đầu')]/following-sibling::*[1]/li/a".decode('utf-8'))
        if len(nodes) == 0:
            lib.Web.removeCache(url)
            tree = lib.Web.load(url).build_tree()
        chapterIdx = 1;
        for node in nodes:
            name = lib.stringify(node)
            link = node.get('href')
            if name == '' or link == '': continue
            name = lib.toUpper(name)
            if '|' in name:
                name = name.split('|')[1].strip()
            link = urljoin(url, link)
            data['chapters'].append({'name': name, 'url': link, 'path': chapterIdx})
            chapterIdx += 1
        return data
    
    
    def getListStory(self):
        storageFolder = "/longhoanggiang/thuvientruyen/"
        lib.makedir(storageFolder)
        url = 'http://truyenngontinh.com/danh-sach-truyen-2/'
        db = MySQL('localhost', 'root', '', 'thuvientruyen')
        db.execute("SET CHARACTER_SET_CLIENT utf8")
        db.commit()
        page = 1
        while page < 1000:
            tree = lib.Web.load(url).build_tree()
            for node in tree.xpath("//table[@class='listing']/tr[position()>2]"):
                aNode = node.xpath(".//th[1]/a")
                name = lib.stringify(aNode).encode('utf-8')
                author = lib.stringify(node.xpath(".//th[2]/a"))
                storyUrl = aNode[0].get('href')
                meta = self.getAStory(storyUrl)
                
                data = {
                    'id': lib.crc32unsigned(storyUrl),
                    'name': name,
                    'author': author,
                    'url': storyUrl,
                    'state': meta['state'],
                    'tag': meta['tag'],
                    'description': meta['desc'],
                    'thumbnail': meta['thumbnail'],
                    'numberOfChapter': len(meta['chapters']),
                    'lastupdate': datetime.datetime.now(),
                    'timestamp': time.time()
                    }
                
                db.insert('story', data)
                db.commit()
                
                # SAVE CHAPTER LIST OF STORY
                lib.makedir(storageFolder + data['id'])
                path = storageFolder + data['id'] + "/data.db"
                lib.file_put_content(json.dumps(meta['chapters']), path)
                print data['id'], data['name'], data['author']
                
#                 path = storageFolder + data['id'] + "/{0}.txt"
#                 fileCount = 1
#                 for chapter in meta['chapters']:
#                     chapterContent = self.getDetail(chapter)
#                     lib.file_put_content(json.dumps(chapterContent), path.format(fileCount))
#                     fileCount += 1
            
            page += 1
            nextNode = tree.xpath("//div[@class='navigation']//div[@class='page_navi']/span[@class='current']/following-sibling::a[1]")
            if len(nextNode) == 0: break
            link = nextNode[0].get('href')
            if link == '': break
            url = link
        
        db.close()
        
    def downloadDetail(self):
        db = MySQL('localhost', 'root', '', 'thuvientruyen')
        rows = db.select('story', ['id', 'name'])
        storageFolder = "/longhoanggiang/thuvientruyen/{0}/data.db"
        for row in rows:
            storyId = row[0]
            storyName = row[1]
            print storyId, storyName
            path = storageFolder.format(storyId)
            data = json.loads(lib.file_get_content(path))
            print data
            
            pool = workerpool.WorkerPool(size=5)
            for chap in data:
                pool.put(WorkerJob(self.getDetail, chap, os.path.dirname(path)))
            pool.shutdown()
            pool.wait()
            
        
if __name__ == '__main__':
    
    t = TruyenNgonTinh()
#     t.getListStory()
    t.downloadDetail()
    logging.info("finished")
    os._exit(1)
            